package com.roy.sparkDemos

import org.apache.spark.{SparkConf, SparkContext}

object SparkWordCount {
  /**
    * spark 提交指令
  * ./spark-submit --master yarn --class "com.roy.spark.examples.JavaWordCount" /home/voltdb/sparkapp/sp_wordcount.jar "hdfs://master:9000/roy/test/test.txt" "hdfs://master:9000/roy/sp_out2"
  * @param args
    */
  def main(args: Array[String]): Unit = {
    if(null == args || args.length != 2){
      println("usage SparkWordCount <inputPath> <outputPath>")
      System.exit(0)
    }
    val scf = new SparkConf()
    scf.setMaster("local[4]")
    scf.setAppName("MySparkDemo")

    val sc = new SparkContext(scf)
    sc.textFile(args(0)).flatMap(line => line.split(" ")).map(word => (word,1))
      .reduceByKey((a,b)=> a+b).saveAsTextFile(args(1))
    sc.stop()
  }
}
